package extractorCode;


import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


public class link2 {
	
	private final static int INITIAL_BUF_SIZE = 65536;
	public static void main(String args[]) throws Exception
	{
	
		int count=0;
	    boolean nextIndex=true;
	    List<Integer>missedPages= new ArrayList();
        List<Integer>noOfpages=new ArrayList();
        // now put in last pages for all keywords
        //noOfpages.add(arg0)
	    BufferedReader reader= new BufferedReader(new FileReader("c:\\newData2\\keywords.txt"));
	    BufferedReader r2= new BufferedReader(new InputStreamReader(System.in));
	    String keyword;
	    int lastIndex;
	    while((keyword=reader.readLine())!=null)
	    {
	    	System.out.println(keyword);
	    	String[] splitarr= keyword.split("\t");
	    	String key= splitarr[0];
	    	int maxPage= Integer.parseInt(splitarr[1]);
	    	count=0;
		    BufferedWriter wr = new BufferedWriter( new FileWriter("C:\\newData2\\link2-"+key+".txt"));
		    BufferedWriter wr2 = new BufferedWriter( new FileWriter("C:\\newData2\\link2-missed pages"+key+".txt"));
		    int i=0;
		  for(i=0;i<=maxPage;i++)
		  {
		System.out.println("page"+i);	
		String page = (fetchPage("http://www.thejakartapost.com/search?search_words="+key+"&submit=GO&per_page="+i));
	    lastIndex= page.indexOf(">Last &rsaquo;</a>");
		
		System.out.println(lastIndex);
		if(page.contains("CAS Authentication wanted!"))
		{
			//Thread.sleep(5000);
			//page = (fetchPage("http://www.thejakartapost.com/search?search_words=fpi&submit=GO&per_page="+i));
		    //missedPages.add(i);
		wr2.write(i+"\t");
		lastIndex=0;
		}
		
		System.out.println(page);
		
		page= page.replaceAll(">\\s*<", "><");
		
		page = page.replaceAll("\t", "");
		page = page.replaceAll("\b", "");
		page = page.replaceAll("\r", "");
		page = page.replaceAll("\n", "");
		page = page.replaceAll("\r\n", "");
		Matcher newsItem = Pattern.compile("<a\\s*href=\"(http://www.thejakartapost.com/news/.*?)\">", Pattern.CASE_INSENSITIVE).matcher( page );
		Matcher newsItem3 = Pattern.compile("<div class=\"date\">(.*?)</div>", Pattern.CASE_INSENSITIVE).matcher( page );
		
	while(newsItem.find() && newsItem3.find())
		{
	//	System.out.println(newsItem.group(1));
		
		String text = newsItem.group(1);
		String date= newsItem3.group(1);
		nextIndex= text.contains("<span>Next</span>");
		System.out.println(text+" "+date);
		text = text.replaceAll("\\<.*?>","");
		text = text.replaceAll("[^\\p{ASCII}]", "");
		
		wr.write(text+"\t"+date+"\n");
		//System.out.println(text);
		
		count++;
		//System.out.println(text);
	//	writer.write("http://newsbusters.org"+newsItem.group(1)+"\n");
		}
	//Thread.sleep(5000);
	//i++;
		}
		  
		 // while(i<6);
		System.out.println("The total number of links "+count);
		
		wr.close();
		wr2.close();
		
	//r2.readLine();
	}
	    }
	//}
	private static String fetchPage(String urlString){
		StringBuilder buffer = new StringBuilder(INITIAL_BUF_SIZE);
		try {
			//URL url = new URL( urlString );
			
			
			 URL url = new URL(urlString);
            HttpURLConnection.setFollowRedirects(false);
            
             HttpURLConnection urlConn = (HttpURLConnection)url.openConnection();

            urlConn.setRequestProperty("User-agent","Mozilla/5.0 (Windows NT 6.1; WOW64; rv:8.0.1) Gecko/20100101 Firefox/8.0.1");
           // urlConn.("Location","http://login.thejakartapost.com:80/login?service=http%3A%2F%2Fwww.thejakartapost.com%2Fsearch%3Fsearch_words%3Dfpi%26submit%3DGO%26per_page%3D5&gateway=true");
            urlConn.setRequestProperty("Cookie","__utma=213011537.1484450262.1319954483.1323468186.1323480089.6; __utmz=213011537.1319954483.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); ci_session=a%3A4%3A%7Bs%3A10%3A%22session_id%22%3Bs%3A32%3A%229c52162d66e4cd4f7e018f6d4e1d3a3e%22%3Bs%3A10%3A%22ip_address%22%3Bs%3A12%3A%2268.99.87.243%22%3Bs%3A10%3A%22user_agent%22%3Bs%3A50%3A%22Mozilla%2F5.0+%28Windows+NT+6.1%3B+WOW64%3B+rv%3A8.0.1%29+Geck%22%3Bs%3A13%3A%22last_activity%22%3Bi%3A1323481333%3B%7Dd6bf58a8607f84a763eda03d37f5dce4; PHPSESSID=rfsu0h2koop5g8audq48qfqdq3; __utmb=213011537.4.10.1323480089; __utmc=213011537");
            urlConn.connect();
             BufferedReader reader = new BufferedReader(new InputStreamReader(urlConn.getInputStream()));
             
			String res=urlConn.getResponseMessage();
			//BufferedInputStream reader = new BufferedInputStream(url.openStream());
			//String url2 = res.indexOf("Location");
			int temp = reader.read();
			while(temp != -1){
				buffer.append((char)temp);
				temp = reader.read();
			}
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return buffer.toString();
	}

}
